import urllib2
from BeautifulSoup import BeautifulSoup
import re

class URLFetcher:
    """Fetch a webpage of the internet (html source)"""
    def __init__(self, url):
        self.url = url
        
    def fetchRaw(self):
        try:
            headers = {"User-Agent": "Creole/0.1a"} # set correct user agent so we can fool website
            request = urllib2.Request(self.url, headers=headers)
            data = urllib2.urlopen(request).read()
            # data = urllib2.urlopen(self.url).read() # use headers and stuff
        except urllib2.HTTPError, e:
            print "HTTP error: %d" % e.code
        except urllib2.URLError, e:
            print "Network error: %s" % e.reason.args[1]
        return data

            

class HTMLCleaner:
    """This class can remove unnecessary tags and strips all tags""" 
    
    def __init__(self, rawHTML):
        self.blacklist = [ 'script', 'style', 'meta', 'link', 'head']
        self.rawHTML = rawHTML
        self.soup = None
        self.clean_html = None
        
    def cleanHTML(self):
        """Cleans the HTML, remove blacklisted tags"""  
        self.soup = BeautifulSoup(''.join(self.rawHTML))
        
        # remove blacklisted tags
        for tag in self.soup.findAll():
            if tag.name.lower() in self.blacklist:
                # blacklisted tags are removed in their entirety
                tag.extract()
                print "extracted " + tag.name.lower()
    
        # clean HTML
        self.clean_html = self.soup.prettify()
        # remove comments
        #p = re.compile( '<!--(.|\n)*-->')
        #self.clean_html = p.sub("",self.clean_html)
    
        # remove DOCTYPE, not working
        #p = re.compile( '<!DOCTYPE(.|\n)*>')
        #clean_html = p.sub("",clean_html)
        
        return self.clean_html
    
    def getSoup(self):
        """Returns the HTML soup"""
        return self.soup
    
    def getPureText(self):
        """Removes all tags from HTML and returns the pure text, also converts htmlentities"""
        #soup = BeautifulSoup(''.join(self.clean_html))
        allText = self.soup.findAll(text=True)
        pureText = map(lambda line: unicode(BeautifulSoup(line,convertEntities=BeautifulSoup.HTML_ENTITIES)), allText)
        #for line in allText:
        #    line = unicode(BeautifulSoup(line,convertEntities=BeautifulSoup.HTML_ENTITIES))
        
        return pureText